{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用python进行文本分类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.准备数据：从文本中构建词向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#词表到向量的转换函数\n",
    "def loadDataSet():\n",
    "    postingList = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],                #切分的词条\n",
    "                 ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],\n",
    "                 ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],\n",
    "                 ['stop', 'posting', 'stupid', 'worthless', 'garbage'],\n",
    "                 ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],\n",
    "                 ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]\n",
    "    classVec = [0,1,0,1,0,1]           #1代表侮辱性文字，0代表正常言论\n",
    "    return postingList,classVec\n",
    "\n",
    "def createVocabList(dataSet):\n",
    "    vocabSet = set([])   #创建一个空集\n",
    "    for document in dataSet:\n",
    "        vocabSet = vocabSet|set(document)  #创建两个集合的并集\n",
    "    return list(vocabSet)\n",
    "\n",
    "def setOfWords2Vec(vocabList, inputSet):\n",
    "    returnVec = [0]*len(vocabList)    #创建一个其中所含元素都为0的向量\n",
    "    for word in inputSet:\n",
    "        if word in vocabList:\n",
    "            returnVec[vocabList.index(word)] = 1\n",
    "        else: pring(\"the word:%s is not in my Vocabulary!\" % word)\n",
    "    return returnVec\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " #### 2.训练算法：从词向量计算概率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\"\"\"\n",
    "函数说明:朴素贝叶斯分类器训练函数\n",
    "\n",
    "Parameters:\n",
    "    trainMatrix - 文档矩阵\n",
    "    trainCategory - 每篇文档类别标签所构成的向量\n",
    "Returns:\n",
    "    p0Vect - 非侮辱类的条件概率数组\n",
    "    p1Vect - 侮辱类的条件概率数组\n",
    "    pAbusive - 文档属于侮辱类的概率\n",
    "\"\"\"\n",
    "def trainNB0(trainMatrix, trainGategory):\n",
    "    numTrainDocs = len(trainMatrix)   #返回训练的文档数目\n",
    "    numWords = len(trainMatrix[0])   #返回每一篇文档的词条数\n",
    "    pAbusive = sum(trainGategory)/float(numTrainDocs)  #文档属于侮辱类的概率\n",
    "    p0Num = np.ones(numWords); p1Num = np.ones(numWords)  #创建numpy.zeros数组,词条出现数初始化为0\n",
    "    p0Denom = 2.0; p1Denom = 2.0   #分母初始化为0\n",
    "    for i in range(numTrainDocs):\n",
    "        if trainGategory[i] == 1:  #统计属于侮辱类的条件概率所需的数据，即P(w0|1),P(w1|1),P(w2|1)···\n",
    "            p1Num += trainMatrix[i]\n",
    "            p1Denom += sum(trainMatrix[i])\n",
    "        else:  #统计属于非侮辱类的条件概率所需的数据，即P(w0|0),P(w1|0),P(w2|0)···\n",
    "            p0Num += trainMatrix[i]\n",
    "            p0Denom += sum(trainMatrix[i])\n",
    "    p1Vect = np.log(p1Num/p1Denom)\n",
    "    p0Vect = np.log(p0Num/p0Denom)\n",
    "    return p0Vect,p1Vect,pAbusive  #返回属于非侮辱类的条件概率数组，属于侮辱类的条件概率数组，文档属于侮辱类的概率\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.测试算法：根据现实情况修改分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "函数说明:朴素贝叶斯分类函数\n",
    "\n",
    "Parameters:\n",
    "    vec2Classify - 待分类的词条数组\n",
    "    p0Vec - 非侮辱类的条件概率数组\n",
    "    p1Vec - 侮辱类的条件概率数组\n",
    "    pClass1 - 文档属于侮辱类的概率\n",
    "Returns:\n",
    "    类别标签 - 1或0\n",
    "\"\"\"\n",
    "def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):\n",
    "    p1 = sum(vec2Classify*p1Vec)+np.log(pClass1)\n",
    "    p0 = sum(vec2Classify*p0Vec)+np.log(1.0-pClass1)\n",
    "    if p1 > p0:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "def testingNB():\n",
    "    listOPosts,listClasses = loadDataSet()\n",
    "    myVocabList = createVocabList(listOPosts)\n",
    "    trainMat = []\n",
    "    for postinDoc in listOPosts:\n",
    "        trainMat.append(setOfWords2Vec(myVocabList, postinDoc))\n",
    "    p0V,p1V,pAb = trainNB0(np.array(trainMat),np.array(listClasses))\n",
    "    testEntry = ['love','my','dalmation']\n",
    "    thisDoc = np.array(setOfWords2Vec(myVocabList,testEntry))\n",
    "    print(testEntry,'classified as:',classifyNB(thisDoc,p0V,p1V,pAb))\n",
    "    testEntry = ['stupid','garbage']\n",
    "    thisDoc = np.array(setOfWords2Vec(myVocabList,testEntry))\n",
    "    print(testEntry,'classified as:',classifyNB(thisDoc,p0V,p1V,pAb))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.准备数据：文档词袋模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "#词袋模型\n",
    "def bagOfWords2VecMN(vocabList, inputSet):\n",
    "    returnVec = [0]*len(vocabList)\n",
    "    for word in inputSet:\n",
    "        if word in vocabList:\n",
    "            returnVec[vocabList.index(word)] += 1\n",
    "    return returnVec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 示例：使用朴素贝叶斯过滤垃圾邮件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#文件解析及完整的垃圾邮件测试函数\n",
    "def textParse(bigString):  #接收一个大字符串并将其解析为字符串列表\n",
    "    import re\n",
    "    listOfTokens = re.split(r'\\W*',bigString)  #利用正则表达式来切分句子\n",
    "    return [tok.lower() for tok in listOfTokens if len(tok) > 2]  #将字符串全部转换为小写，并去除空格以及长度小于3的字符串\n",
    "\n",
    "def spamTest():\n",
    "    docList=[]; classList=[]; fullText=[]\n",
    "    for i in range(1,26):   #遍历25个文件，range(1,26)代表从1到26，不包含26.\n",
    "        wordList = textParse(open('data/email/spam/%d.txt'%i).read()) #读取每个垃圾邮件，并将字符串转换成字符串列表\n",
    "        docList.append(wordList)\n",
    "        #fullText.extend(wordList)  #不明白用来干嘛的？注释掉也没有影响\n",
    "        classList.append(1)  #标记垃圾邮件，1表示垃圾文件\n",
    "        wordList = textParse(open('data/email/ham/%d.txt' % i).read())  #读取每个非垃圾邮件，并将字符串转换成字符串列表\n",
    "        docList.append(wordList)\n",
    "        #fullText.extend(wordList)#不明白用来干嘛的？注释掉也没有影响\n",
    "        classList.append(0)  #标记非垃圾邮件，0表示非垃圾文件\n",
    "    vocabList = createVocabList(docList)  #创建词汇表，不重复\n",
    "    trainingSet = list(range(50));  testSet=[]  #创建存储训练集的索引值的列表和测试集的索引值的列表\n",
    "    for i in range(10):  #从50个邮件中，随机挑选出40个作为训练集,10个做测试集\n",
    "        randIndex = int(np.random.uniform(0,len(trainingSet)))  #随机选取索索引值\n",
    "        testSet.append(trainingSet[randIndex])  #添加测试集的索引值\n",
    "        del(trainingSet[randIndex])  #在训练集列表中删除添加到测试集的索引值\n",
    "    trainMat = []; trainClasses = []  #创建训练集矩阵和训练集类别标签系向量\n",
    "    for docIndex in trainingSet:  #遍历训练集\n",
    "        trainMat.append(setOfWords2Vec(vocabList,docList[docIndex]))  #将生成的词集模型添加到训练矩阵中\n",
    "        trainClasses.append(classList[docIndex])  #将类别添加到训练集类别标签系向量中\n",
    "    p0V,p1V,pSpam = trainNB0(np.array(trainMat),np.array(trainClasses))  #训练朴素贝叶斯模型\n",
    "    errorCount = 0  #错误分类计数\n",
    "    for docIndex in testSet:  #遍历测试集\n",
    "        wordVector = setOfWords2Vec(vocabList,docList[docIndex])  #测试集的词集模型\n",
    "        if classifyNB(np.array(wordVector),p0V,p1V,pSpam) != classList[docIndex]:  #如果分类错误\n",
    "            errorCount +=1  #错误计数加1\n",
    "            print(' classification error:',docList[docIndex])\n",
    "    print(' the error rate is:', float(errorCount)/len(testSet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " classification error: ['home', 'based', 'business', 'opportunity', 'knocking', 'your', 'door', 'don抰', 'rude', 'and', 'let', 'this', 'chance', 'you', 'can', 'earn', 'great', 'income', 'and', 'find', 'your', 'financial', 'life', 'transformed', 'learn', 'more', 'here', 'your', 'success', 'work', 'from', 'home', 'finder', 'experts']\n",
      " the error rate is: 0.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\re.py:212: FutureWarning: split() requires a non-empty pattern match.\n",
      "  return _compile(pattern, flags).split(string, maxsplit)\n"
     ]
    }
   ],
   "source": [
    "spamTest()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " classification error: ['scifinance', 'now', 'automatically', 'generates', 'gpu', 'enabled', 'pricing', 'risk', 'model', 'source', 'code', 'that', 'runs', '300x', 'faster', 'than', 'serial', 'code', 'using', 'new', 'nvidia', 'fermi', 'class', 'tesla', 'series', 'gpu', 'scifinance', 'derivatives', 'pricing', 'and', 'risk', 'model', 'development', 'tool', 'that', 'automatically', 'generates', 'and', 'gpu', 'enabled', 'source', 'code', 'from', 'concise', 'high', 'level', 'model', 'specifications', 'parallel', 'computing', 'cuda', 'programming', 'expertise', 'required', 'scifinance', 'automatic', 'gpu', 'enabled', 'monte', 'carlo', 'pricing', 'model', 'source', 'code', 'generation', 'capabilities', 'have', 'been', 'significantly', 'extended', 'the', 'latest', 'release', 'this', 'includes']\n",
      " the error rate is: 0.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\re.py:212: FutureWarning: split() requires a non-empty pattern match.\n",
      "  return _compile(pattern, flags).split(string, maxsplit)\n"
     ]
    }
   ],
   "source": [
    "spamTest()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
